In this particular task, I was working with a Convolutional Neural Network (CNN) that had previously been trained on a dataset containing images of cats and dogs. However, as I attempted to retrain and test the model using this dataset, I found that it was taking a significant amount of time to complete these processes.
As such, I decided to switch to a different dataset that would be quicker to process. I opted for a dataset of fish images, which took only around half an hour to complete. This allowed me to move forward with my analysis much more efficiently.
import numpy as np
import pandas as pd
from pathlib import Path
import os.path
import matplotlib.pyplot as plt
from IPython.display import Image, display
import matplotlib.cm as cm
from sklearn.model_selection import train_test_split
import tensorflow as tf
image_dir = Path('Fish_Dataset/Fish_Dataset')
# Get filepaths and labels
filepaths = list(image_dir.glob(r'**/*.png'))
labels = list(map(lambda x: os.path.split(os.path.split(x)[0])[1], filepaths))
filepaths = pd.Series(filepaths, name='Filepath').astype(str)
labels = pd.Series(labels, name='Label')
# Concatenate filepaths and labels
image_df = pd.concat([filepaths, labels], axis=1)
# Drop GT images
image_df = image_df[image_df['Label'].apply(lambda x: x[-2:] != 'GT')]
# Activate this code to use only 100 pictures for each label
# lst = []
# for l in image_df['Label'].unique():
# lst.append(image_df[image_df['Label'] == l] .sample(100, random_state = 0))
# # Concatenate the DataFrames
# image_df = pd.concat(lst)
# Shuffle the DataFrame and reset index
image_df = image_df.sample(frac=1).reset_index(drop = True)
# Show the result
image_df.head(3)
| Filepath | Label | |
|---|---|---|
| 0 | Fish_Dataset\Fish_Dataset\Shrimp\Shrimp\00296.png | Shrimp |
| 1 | Fish_Dataset\Fish_Dataset\Black Sea Sprat\Blac... | Black Sea Sprat |
| 2 | Fish_Dataset\Fish_Dataset\Gilt-Head Bream\Gilt... | Gilt-Head Bream |
# Display 20 picture of the dataset with their labels
fig, axes = plt.subplots(nrows=3, ncols=5, figsize=(15, 7),
subplot_kw={'xticks': [], 'yticks': []})
for i, ax in enumerate(axes.flat):
ax.imshow(plt.imread(image_df.Filepath[i]))
ax.set_title(image_df.Label[i])
plt.tight_layout()
plt.show()
# Separate in train and test data
train_df, test_df = train_test_split(image_df, train_size=0.9, shuffle=True, random_state=1)
train_generator = tf.keras.preprocessing.image.ImageDataGenerator(
preprocessing_function=tf.keras.applications.mobilenet_v2.preprocess_input,
validation_split=0.2
)
test_generator = tf.keras.preprocessing.image.ImageDataGenerator(
preprocessing_function=tf.keras.applications.mobilenet_v2.preprocess_input
)
train_images = train_generator.flow_from_dataframe(
dataframe=train_df,
x_col='Filepath',
y_col='Label',
target_size=(224, 224),
color_mode='rgb',
class_mode='categorical',
batch_size=32,
shuffle=True,
seed=42,
subset='training'
)
val_images = train_generator.flow_from_dataframe(
dataframe=train_df,
x_col='Filepath',
y_col='Label',
target_size=(224, 224),
color_mode='rgb',
class_mode='categorical',
batch_size=32,
shuffle=True,
seed=42,
subset='validation'
)
test_images = test_generator.flow_from_dataframe(
dataframe=test_df,
x_col='Filepath',
y_col='Label',
target_size=(224, 224),
color_mode='rgb',
class_mode='categorical',
batch_size=32,
shuffle=False
)
Found 6480 validated image filenames belonging to 9 classes. Found 1620 validated image filenames belonging to 9 classes. Found 900 validated image filenames belonging to 9 classes.
# Load the pretained model
pretrained_model = tf.keras.applications.MobileNetV2(
input_shape=(224, 224, 3),
include_top=False,
weights='imagenet',
pooling='avg'
)
pretrained_model.trainable = False
inputs = pretrained_model.input
x = tf.keras.layers.Dense(128, activation='relu')(pretrained_model.output)
x = tf.keras.layers.Dense(128, activation='relu')(x)
outputs = tf.keras.layers.Dense(9, activation='softmax')(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy']
)
history = model.fit(
train_images,
validation_data=val_images,
epochs=5,
callbacks=[
tf.keras.callbacks.EarlyStopping(
monitor='val_loss',
patience=1,
restore_best_weights=True
)
]
)
Epoch 1/5 203/203 [==============================] - 172s 823ms/step - loss: 0.1699 - accuracy: 0.9519 - val_loss: 0.0364 - val_accuracy: 0.9883 Epoch 2/5 203/203 [==============================] - 165s 813ms/step - loss: 0.0165 - accuracy: 0.9954 - val_loss: 0.0197 - val_accuracy: 0.9920 Epoch 3/5 203/203 [==============================] - 138s 679ms/step - loss: 0.0076 - accuracy: 0.9985 - val_loss: 0.0065 - val_accuracy: 0.9975 Epoch 4/5 203/203 [==============================] - 143s 706ms/step - loss: 0.0020 - accuracy: 0.9994 - val_loss: 0.0206 - val_accuracy: 0.9907
pd.DataFrame(history.history)[['accuracy','val_accuracy']].plot()
plt.title("Accuracy")
plt.show()
pd.DataFrame(history.history)[['loss','val_loss']].plot()
plt.title("Loss")
plt.show()
results = model.evaluate(test_images, verbose=0)
print(" Test Loss: {:.5f}".format(results[0]))
print("Test Accuracy: {:.2f}%".format(results[1] * 100))
Test Loss: 0.00424 Test Accuracy: 99.89%
# Predict the label of the test_images
pred = model.predict(test_images)
pred = np.argmax(pred,axis=1)
# Map the label
labels = (train_images.class_indices)
labels = dict((v,k) for k,v in labels.items())
pred = [labels[k] for k in pred]
# Display the result
print(f'The first 5 predictions: {pred[:5]}')
29/29 [==============================] - 22s 702ms/step The first 5 predictions: ['Black Sea Sprat', 'Red Sea Bream', 'Striped Red Mullet', 'Gilt-Head Bream', 'Red Mullet']
from sklearn.metrics import classification_report
y_test = list(test_df.Label)
print(classification_report(y_test, pred))
precision recall f1-score support
Black Sea Sprat 1.00 1.00 1.00 115
Gilt-Head Bream 1.00 1.00 1.00 84
Hourse Mackerel 1.00 1.00 1.00 86
Red Mullet 1.00 1.00 1.00 96
Red Sea Bream 1.00 1.00 1.00 107
Sea Bass 1.00 0.99 0.99 98
Shrimp 1.00 1.00 1.00 103
Striped Red Mullet 0.99 1.00 1.00 104
Trout 1.00 1.00 1.00 107
accuracy 1.00 900
macro avg 1.00 1.00 1.00 900
weighted avg 1.00 1.00 1.00 900
from sklearn.metrics import confusion_matrix
import seaborn as sns
cf_matrix = confusion_matrix(y_test, pred, normalize='true')
plt.figure(figsize = (10,6))
sns.heatmap(cf_matrix, annot=True, xticklabels = sorted(set(y_test)), yticklabels = sorted(set(y_test)))
plt.title('Normalized Confusion Matrix')
plt.show()
# Display 15 picture of the dataset with their labels
fig, axes = plt.subplots(nrows=3, ncols=5, figsize=(15, 7),
subplot_kw={'xticks': [], 'yticks': []})
for i, ax in enumerate(axes.flat):
ax.imshow(plt.imread(test_df.Filepath.iloc[i]))
ax.set_title(f"True: {test_df.Label.iloc[i]}\nPredicted: {pred[i]}")
plt.tight_layout()
plt.show()
def get_img_array(img_path, size):
img = tf.keras.preprocessing.image.load_img(img_path, target_size=size)
array = tf.keras.preprocessing.image.img_to_array(img)
# We add a dimension to transform our array into a "batch"
# of size "size"
array = np.expand_dims(array, axis=0)
return array
def make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):
# First, we create a model that maps the input image to the activations
# of the last conv layer as well as the output predictions
grad_model = tf.keras.models.Model(
[model.inputs], [model.get_layer(last_conv_layer_name).output, model.output]
)
# Then, we compute the gradient of the top predicted class for our input image
# with respect to the activations of the last conv layer
with tf.GradientTape() as tape:
last_conv_layer_output, preds = grad_model(img_array)
if pred_index is None:
pred_index = tf.argmax(preds[0])
class_channel = preds[:, pred_index]
# This is the gradient of the output neuron (top predicted or chosen)
# with regard to the output feature map of the last conv layer
grads = tape.gradient(class_channel, last_conv_layer_output)
# This is a vector where each entry is the mean intensity of the gradient
# over a specific feature map channel
pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))
# We multiply each channel in the feature map array
# by "how important this channel is" with regard to the top predicted class
# then sum all the channels to obtain the heatmap class activation
last_conv_layer_output = last_conv_layer_output[0]
heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
heatmap = tf.squeeze(heatmap)
# For visualization purpose, we will also normalize the heatmap between 0 & 1
heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
return heatmap.numpy()
def save_and_display_gradcam(img_path, heatmap, cam_path="cam.jpg", alpha=0.4):
# Load the original image
img = tf.keras.preprocessing.image.load_img(img_path)
img = tf.keras.preprocessing.image.img_to_array(img)
# Rescale heatmap to a range 0-255
heatmap = np.uint8(255 * heatmap)
# Use jet colormap to colorize heatmap
jet = cm.get_cmap("jet")
# Use RGB values of the colormap
jet_colors = jet(np.arange(256))[:, :3]
jet_heatmap = jet_colors[heatmap]
# Create an image with RGB colorized heatmap
jet_heatmap = tf.keras.preprocessing.image.array_to_img(jet_heatmap)
jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))
jet_heatmap = tf.keras.preprocessing.image.img_to_array(jet_heatmap)
# Superimpose the heatmap on original image
superimposed_img = jet_heatmap * alpha + img
superimposed_img = tf.keras.preprocessing.image.array_to_img(superimposed_img)
# Save the superimposed image
superimposed_img.save(cam_path)
# Display Grad CAM
# display(Image(cam_path))
return cam_path
preprocess_input = tf.keras.applications.mobilenet_v2.preprocess_input
decode_predictions = tf.keras.applications.mobilenet_v2.decode_predictions
last_conv_layer_name = "Conv_1"
img_size = (224,224)
# Remove last layer's softmax
model.layers[-1].activation = None
# Display the part of the pictures used by the neural network to classify the pictures
fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(15, 10),
subplot_kw={'xticks': [], 'yticks': []})
for i, ax in enumerate(axes.flat):
img_path = test_df.Filepath.iloc[i]
img_array = preprocess_input(get_img_array(img_path, size=img_size))
heatmap = make_gradcam_heatmap(img_array, model, last_conv_layer_name)
cam_path = save_and_display_gradcam(img_path, heatmap)
ax.imshow(plt.imread(cam_path))
ax.set_title(f"True: {test_df.Label.iloc[i]}\nPredicted: {pred[i]}")
plt.tight_layout()
plt.show()
One of the key insights that I was able to glean from this analysis was that the CNN was highly focused on the distinguishing features of the fish images. These included traits like spots, unique shapes, and other defining characteristics. By taking these features into account, the model was able to accurately differentiate between the various types of fish in the dataset.
Overall, this analysis was an important step in better understanding the capabilities and limitations of this particular CNN, as well as gaining insights into how it can be trained more effectively in the future.